import os.path
import re
from time import sleep

import requests

from submodules.common.lib import local_paths
from submodules.common.lib.json_ import dict_to_json_file, json_file_to_dict
from submodules.common.lib.log_ import get_logger
from submodules.common.lib.url_ import set_url_param, url_to_dict

logger = get_logger()


def get_sse_etf_list():
    req_info = {
        "headers": {
            "Host": "query.sse.com.cn",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36",
            "Accept": "*/*",
            "Referer": "http://www.sse.com.cn/",
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
            "Cookie": 'yfx_c_g_u_id_10000042=_ck22030218260612603837132545667; yfx_mr_10000042=::market_type_free_search::::baidu::::::::www.baidu.com::::pmf_from_free_search; yfx_key_10000042=; VISITED_MENU=["8482"]; gdp_user_id=gioenc-c07abdcg,5g2d,5ddb,8464,dg83g491845e; ba17301551dcbaf9_gdp_session_id=aa23d7cf-518c-4f95-b8bb-d90ba28e249e; yfx_f_l_v_t_10000042=f_t_1646216766238__r_t_1666594281004__v_t_1666594281004__r_c_1; ba17301551dcbaf9_gdp_session_id_aa23d7cf-518c-4f95-b8bb-d90ba28e249e=true',
        },
        "params": {
            "isPagination": "false",
            "pageHelp.pageSize": "100",
            "pageHelp.pageNo": "1",
            "pageHelp.beginPage": "1",
            "pageHelp.cacheSize": "1",
            "pageHelp.endPage": "1",
            "pagecache": "false",
            "sqlId": "FUND_LIST",
            "fundType": "00",
            "subClass": "01",
            "_": "1666594281504",
        },
        "data": {},
        "cookies": {
            "fx_c_g_u_id_10000042": "_ck22030218260612603837132545667",
            "yfx_mr_10000042": "::market_type_free_search::::baidu::::::::www.baidu.com::::pmf_from_free_search",
            "yfx_key_10000042": "",
            "VISITED_MENU": '["8482"]',
            "gdp_user_id": "gioenc-c07abdcg,5g2d,5ddb,8464,dg83g491845e",
            "ba17301551dcbaf9_gdp_session_id": "aa23d7cf-518c-4f95-b8bb-d90ba28e249e",
            "yfx_f_l_v_t_10000042": "f_t_1646216766238__r_t_1666594281004__v_t_1666594281004__r_c_1",
            "ba17301551dcbaf9_gdp_session_id_aa23d7cf-518c-4f95-b8bb-d90ba28e249e": "tru",
        },
        "url": "http://query.sse.com.cn/commonSoaQuery.do?isPagination=false&pageHelp.pageSize=100&pageHelp.pageNo=1&pageHelp.beginPage=1&pageHelp.cacheSize=1&pageHelp.endPage=1&pagecache=false&sqlId=FUND_LIST&fundType=00&subClass=01&_=1666594281504",
        "path": "/commonSoaQuery.do",
    }

    url = req_info["url"]
    sub_class_list = ["01", "03", "08", "09", "31", "33", "06"]

    etf_info_list = []
    for sub_class in sub_class_list:
        params_dict = {
            "subClass": sub_class,
        }
        url = set_url_param(url, params_dict)
        headers = req_info["headers"]
        cookies = req_info["cookies"]

        res = requests.get(url=url, headers=headers, cookies=cookies, verify=False)
        print(res.url)
        try:
            etf_info_list += res.json()["result"]
        except Exception as e:
            logger.error(repr(e))
            break
    sse_etf_info_path = os.path.join(local_paths.project_tmp_path, "sse_etf_info.json")
    dict_to_json_file(etf_info_list, sse_etf_info_path)
    return etf_info_list


black_list_fund_code = [
    "513730",  # 东南亚
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #
    "159650",  #

    "159650",
    "159651",
    "159821",
    "515600",
    "510010",
    "512870",
    "510110",
    "159001",
    "512920",
    "510440",
    "515680",
    "510770",
    "159962",
    "159815",
    "515760",
    "159903",
    "159706",
    "512650",
    "515500",
    "510020",
    "510210",
    "517000",
    "159974",
    "517850",
    "159623",
    "517500",
    "513300",
    "517900",
    "516210",
    "515290",
    "517170",
    "516700",
    "588060",
    "588280",
    "516980",
    "159632",
    "513280",
    "159987",
    "517060",
    "516190",
    "560000",
    "159979",
    "517550",
    "517550",
    "510220",
    "159731",
    "517360",
    "510060",
    "159965",
    "516600",
    "515090",
    "515200",
    "510160",
    "159743",
    "515660",
]


def process_sse_etf_info():
    final_list = []
    tag_list = []

    sse_etf_info_path = os.path.join(local_paths.project_tmp_path, "sse_etf_info.json")
    try:
        sse_etf_info_dict = json_file_to_dict(sse_etf_info_path)
    except:
        sse_etf_info_dict = {}
    sse_etf_info_dict_sorted = sorted(sse_etf_info_dict, key=lambda x: x["fundAbbr"])
    for item in sse_etf_info_dict_sorted:
        fundCode = item["fundCode"]
        fund_abbr = item["fundAbbr"]
        if fundCode in black_list_fund_code:
            fund_abbr = ""
        tag = re.sub(r"[0-9a-zA-Z]", "", fund_abbr)
        tag = tag.replace("平安金", "")
        tag = tag.replace("基金", "")
        tag = tag.replace("上海", "")
        tag = tag.replace("中证", "")
        tag = tag.replace("上证", "")
        tag = tag.replace("华夏", "")
        tag = tag.replace("中银", "")
        tag = tag.replace("易方达", "")
        tag = tag.replace("华安", "")
        tag = tag.replace("华夏", "")
        tag = tag.replace("南方", "")
        tag = tag.replace("博时", "")
        tag = tag.replace("嘉实", "")
        tag = tag.replace("天弘", "")
        tag = tag.replace("富国", "")
        tag = tag.replace("工银", "")
        tag = tag.replace("广发", "")
        tag = tag.replace("建信", "")
        tag = tag.replace("融通", "")
        tag = tag.replace("广发", "")
        tag = tag.replace("方达", "")
        tag = tag.replace("万家", "")
        tag = tag.replace("国寿", "")
        tag = tag.replace("国泰", "")
        tag = tag.replace("东证", "")
        tag = tag.replace("南方", "")
        tag = tag.replace("泰康", "")
        tag = tag.replace("之江凤凰", "")
        tag = tag.replace("兴业", "")
        tag = tag.replace("国信", "")
        tag = tag.replace("山证", "")
        tag = tag.replace("平安", "")
        tag = tag.replace("方正", "")
        tag = tag.replace("景顺", "")
        tag = tag.replace("永赢", "")
        tag = tag.replace("浦银", "")
        tag = tag.replace("添富", "")
        tag = tag.replace("漂亮", "")
        tag = tag.replace("招商", "")
        tag = tag.replace("博时", "")
        tag = tag.replace("浦银", "")
        tag = tag.replace("浦银", "")
        tag = tag.replace("鹏华", "")
        tag = tag.replace("国投", "")
        tag = tag.replace("国证", "")
        tag = tag.replace("中融", "")
        tag = tag.replace("指数", "")
        tag = tag.replace("快钱", "")
        tag = tag.replace("快线", "")
        tag = tag.replace("国际", "")
        tag = tag.replace("龙头", "")
        tag = tag.replace("中创", "")
        tag = tag.replace("中盘", "")
        tag = tag.replace("龙头", "")
        tag = tag.replace("银基", "")
        tag = tag.replace("创创", "")
        tag = tag.replace("创业大盘", "")
        tag = tag.replace("京津冀基", "")
        tag = tag.replace("新材料基", "")
        tag = tag.replace("易基", "")
        tag = tag.replace("科创基", "")
        tag = tag.replace("银行指基", "")
        tag = tag.replace("银行股基", "")
        tag = tag.replace("中国", "")
        tag = tag.replace("价值", "")
        tag = tag.replace("低波", "")
        tag = tag.replace("共赢", "")
        tag = tag.replace("开放", "")
        tag = tag.replace("指增", "")
        tag = tag.replace("核心", "")
        tag = tag.replace("富邦", "")
        tag = tag.replace("添企", "")
        tag = tag.replace("银华", "")
        tag = tag.replace("增强", "")
        tag = tag.replace("产业", "")
        tag = tag.replace("行业", "")
        tag = tag.replace("非周", "")
        tag = tag.replace("质量", "")
        tag = tag.replace("高股息", "")
        tag = tag.replace("红利", "")
        tag = tag.replace("新华", "")
        tag = tag.replace("成长", "")

        tag = tag.replace("恒指", "港股")
        tag = tag.replace("有色金属", "有色")
        tag = tag.replace("消电", "消费电子")
        tag = tag.replace("新能源", "新能")
        tag = tag.replace("新能", "新能源")
        tag = tag.replace("沪港深", "中华")
        tag = tag.replace("香港", "港股")
        tag = tag.replace("港股", "港")
        tag = tag.replace("港", "港股")
        tag = tag.replace("恒生", "港股")
        tag = tag.replace("升级", "产业升级")
        tag = tag.replace("生科", "生物科技")
        tag = tag.replace("深证成指", "深成")
        tag = tag.replace("新材", "新材料")
        tag = tag.replace("消服", "消费服务")
        tag = tag.replace("港股通", "")

        tag = tag.replace("港股综", "")
        tag = tag.replace("科技港股", "港股科技")
        tag = tag.replace("生物港股科技", "港股生物科技")
        tag = tag.replace("粤港股澳大湾区", "粤港澳大湾区")
        tag = tag.replace("医服", "医疗服务")
        tag = tag.replace("科业", "科创创业")
        tag = tag.replace("科创板", "科创")
        tag = tag.replace("稀金属", "")
        tag = tag.replace("股消费", "港股消费")
        tag = tag.replace("综指", "上证")
        tag = tag.replace("创新药", "新药")
        tag = tag.replace("新药", "创新药")
        tag = tag.replace("创新药企", "")
        tag = tag.replace("民企领先", "")

        if "湾区" in fund_abbr:
            tag = ""
        if "长江" in fund_abbr:
            tag = ""
        if "医疗" in fund_abbr:
            tag = "医疗"
        if "医药" in fund_abbr:
            tag = "医药"
        if "黄金" in fund_abbr:
            tag = "金"
        if "碳中和" in fund_abbr:
            tag = "碳中和"
        if "生物" == tag:
            tag = "生物科技"
        if fundCode == "516570":
            tag = "化工"
        if fundCode == "516790":
            tag = "医疗保健"
        item.update({"tag": tag})
    for item in sse_etf_info_dict_sorted:
        tag = item["tag"]
        if len(tag) > 1 and tag not in tag_list:
            final_list.append(item)
            tag_list.append(tag)
    return final_list


def get_szse_etf_list():
    req_info = {
        "headers": {
            "Host": "www.szse.cn",
            "Accept": "application/json, text/javascript, */*; q=0.01",
            "X-Request-Type": "ajax",
            "X-Requested-With": "XMLHttpRequest",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36",
            "Content-Type": "application/json",
            "Referer": "http://www.szse.cn/market/product/list/etfList/index.html",
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
        },
        "params": {
            "SHOWTYPE": "JSON",
            "CATALOGID": "1945",
            "tab1PAGENO": "1",
            "random": "0.9249280647700491",
        },
        "data": {},
        "cookies": {},
        "url": "http://www.szse.cn/api/report/ShowReport/data?SHOWTYPE=JSON&CATALOGID=1945&tab1PAGENO=1&random=0.9249280647700491",
        "path": "/api/report/ShowReport/data",
    }

    # url1 = f"http://query.sse.com.cn/commonSoaQuery.do?jsonCallBack=jsonpCallback20163472&isPagination=true&pageHelp.pageSize=25&pageHelp.pageNo=1&pageHelp.beginPage=1&pageHelp.cacheSize=1&pageHelp.endPage=1&pagecache=false&sqlId=FUND_LIST&fundType=00&subClass=01&_=1666294854949"
    # res = requests.get(url=url1, headers=headers, params=params, verify=False)

    url = req_info["url"]

    etf_info_list = []
    for i in range(1, 1000):
        params_dict = {
            "PAGENO": f"{i}",
        }
        url = set_url_param(url, params_dict)
        headers = req_info["headers"]
        cookies = req_info["cookies"]
        has_res = False
        for try_times in range(5):
            try:
                res = requests.get(url=url, headers=headers, cookies=cookies, verify=False)
                print(res.url)
                data = res.json()[0]["data"]
                has_res = True
            except:
                has_res = False
            if has_res:
                break
            sleep(3)
        if len(data) > 0:
            etf_info_list += data
        else:
            break
        sleep(1)
    etf_info_path = os.path.join(local_paths.project_tmp_path, "szse_etf_info.json")
    dict_to_json_file(etf_info_list, etf_info_path)
    return etf_info_list


def process_szse_etf_info():
    etf_info_path = os.path.join(local_paths.project_tmp_path, "szse_etf_info.json")
    try:
        etf_info_dict = json_file_to_dict(etf_info_path)
    except:
        etf_info_dict = {}
    preprocess_list = []
    for item in etf_info_dict:
        tmp_list = item["kzjcurl"].split("'")
        url_dict = url_to_dict(tmp_list[1])
        preprocess_list.append(
            {
                "fundCode": url_dict["params"]["stock"],
                "fundAbbr": url_dict["params"]["name"],
            }
        )

    final_list = []
    tag_list = []

    etf_info_dict_sorted = sorted(preprocess_list, key=lambda x: x["fundAbbr"])

    for item in etf_info_dict_sorted:
        fundCode = item["fundCode"]
        fund_abbr = item["fundAbbr"]
        if fundCode in black_list_fund_code:
            fund_abbr = ""
        tag = re.sub(r"[0-9a-zA-Z]", "", fund_abbr)
        tag = tag.replace("平安金", "")
        tag = tag.replace("基金", "")
        tag = tag.replace("上海", "")
        tag = tag.replace("中证", "")
        tag = tag.replace("上证", "")
        tag = tag.replace("华夏", "")
        tag = tag.replace("中银", "")
        tag = tag.replace("易方达", "")
        tag = tag.replace("华安", "")
        tag = tag.replace("华夏", "")
        tag = tag.replace("南方", "")
        tag = tag.replace("博时", "")
        tag = tag.replace("嘉实", "")
        tag = tag.replace("天弘", "")
        tag = tag.replace("富国", "")
        tag = tag.replace("工银", "")
        tag = tag.replace("广发", "")
        tag = tag.replace("建信", "")
        tag = tag.replace("融通", "")
        tag = tag.replace("广发", "")
        tag = tag.replace("方达", "")
        tag = tag.replace("万家", "")
        tag = tag.replace("国寿", "")
        tag = tag.replace("国泰", "")
        tag = tag.replace("东证", "")
        tag = tag.replace("南方", "")
        tag = tag.replace("泰康", "")
        tag = tag.replace("之江凤凰", "")
        tag = tag.replace("兴业", "")
        tag = tag.replace("国信", "")
        tag = tag.replace("山证", "")
        tag = tag.replace("平安", "")
        tag = tag.replace("方正", "")
        tag = tag.replace("景顺", "")
        tag = tag.replace("永赢", "")
        tag = tag.replace("浦银", "")
        tag = tag.replace("添富", "")
        tag = tag.replace("漂亮", "")
        tag = tag.replace("招商", "")
        tag = tag.replace("博时", "")
        tag = tag.replace("浦银", "")
        tag = tag.replace("浦银", "")
        tag = tag.replace("鹏华", "")
        tag = tag.replace("国投", "")
        tag = tag.replace("国证", "")
        tag = tag.replace("中融", "")
        tag = tag.replace("指数", "")
        tag = tag.replace("快钱", "")
        tag = tag.replace("快线", "")
        tag = tag.replace("国际", "")
        tag = tag.replace("龙头", "")
        tag = tag.replace("中创", "")
        tag = tag.replace("中盘", "")
        tag = tag.replace("龙头", "")
        tag = tag.replace("银基", "")
        tag = tag.replace("创创", "")
        tag = tag.replace("创业大盘", "")
        tag = tag.replace("京津冀基", "")
        tag = tag.replace("新材料基", "")
        tag = tag.replace("易基", "")
        tag = tag.replace("科创基", "")
        tag = tag.replace("银行指基", "")
        tag = tag.replace("银行股基", "")
        tag = tag.replace("中国", "")
        tag = tag.replace("价值", "")
        tag = tag.replace("低波", "")
        tag = tag.replace("共赢", "")
        tag = tag.replace("开放", "")
        tag = tag.replace("指增", "")
        tag = tag.replace("核心", "")
        tag = tag.replace("富邦", "")
        tag = tag.replace("添企", "")
        tag = tag.replace("银华", "")
        tag = tag.replace("增强", "")
        tag = tag.replace("产业", "")
        tag = tag.replace("行业", "")
        tag = tag.replace("非周", "")
        tag = tag.replace("质量", "")
        tag = tag.replace("高股息", "")
        tag = tag.replace("红利", "")
        tag = tag.replace("新华", "")
        tag = tag.replace("成长", "")

        tag = tag.replace("恒指", "港股")
        tag = tag.replace("有色金属", "有色")
        tag = tag.replace("消电", "消费电子")
        tag = tag.replace("新能源", "新能")
        tag = tag.replace("新能", "新能源")
        tag = tag.replace("沪港深", "中华")
        tag = tag.replace("香港", "港股")
        tag = tag.replace("港股", "港")
        tag = tag.replace("港", "港股")
        tag = tag.replace("恒生", "港股")
        tag = tag.replace("升级", "产业升级")
        tag = tag.replace("生科", "生物科技")
        tag = tag.replace("深证成指", "深成")

        tag = tag.replace("新材", "新材料")
        tag = tag.replace("消服", "消费服务")
        tag = tag.replace("港股通", "")
        tag = tag.replace("港股综", "")
        tag = tag.replace("科技港股", "港股科技")
        tag = tag.replace("生物港股科技", "港股生物科技")
        tag = tag.replace("粤港股澳大湾区", "粤港澳大湾区")
        tag = tag.replace("医服", "医疗服务")
        tag = tag.replace("科业", "科创创业")
        tag = tag.replace("科创板", "科创")
        tag = tag.replace("稀金属", "")
        tag = tag.replace("股消费", "港股消费")
        tag = tag.replace("综指", "上证")
        tag = tag.replace("创新药", "新药")
        tag = tag.replace("新药", "创新药")
        tag = tag.replace("创新药企", "")
        tag = tag.replace("民企领先", "")
        if "湾区" in fund_abbr:
            tag = ""
        if "长江" in fund_abbr:
            tag = ""
        if "医疗" in fund_abbr:
            tag = "医疗"
        if "医药" in fund_abbr:
            tag = "医药"
        if "黄金" in fund_abbr:
            tag = "金"
        if "碳中和" in fund_abbr:
            tag = "碳中和"
        if "生物" == tag:
            tag = "生物科技"
        if fundCode == "516570":
            tag = "化工"
        item.update({"tag": tag})
    for item in etf_info_dict_sorted:
        tag = item["tag"]
        if len(tag) > 1 and tag not in tag_list and "债" not in tag:
            final_list.append(item)
            tag_list.append(tag)
    return final_list


def get_etf_list():
    """
    [{'fundCode': '159001', 'fundAbbr': '货币ETF', 'tag': '货币'},...]
    """
    get_szse_etf_list()
    get_sse_etf_list()

    sse_etf_list = process_sse_etf_info()
    szse_etf_list = process_szse_etf_info()
    etf_list_all = sse_etf_list + szse_etf_list
    etf_list_all_process = []
    etf_list_all_tag = []
    for item in etf_list_all:
        tag = item["tag"]
        if tag not in etf_list_all_tag:
            etf_list_all_tag.append(tag)
            etf_list_all_process.append(
                {
                    "fundCode": item["fundCode"],
                    "fundAbbr": item["fundAbbr"],
                    "tag": item["tag"],
                }
            )
    etf_list_all_process_sorted = sorted(etf_list_all_process, key=lambda x: x["tag"])
    etf_list_path = os.path.join(local_paths.project_tmp_path, "etf_list.txt")

    f = open(etf_list_path, "w", encoding="utf-8")
    for item in etf_list_all_process_sorted:
        f.write(item["fundCode"] + "\t" + item["tag"] + "\n")
    f.close()
    return etf_list_all_process_sorted


if __name__ == "__main__":
    etf_list = get_etf_list()
    for item in etf_list:
        print(item)
    print(len(etf_list))
